package com.nlp.visualization.crawler.pipeline;

import com.alibaba.fastjson.JSON;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.utils.FilePersistentBase;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;

/**
 * @author LXM
 * @Title: JsonAsFilePipeline
 * @Description: 将爬取的网页以json的形式保存
 * @date 2018/3/14下午5:28
 */
public class JsonAsFilePipeline implements Pipeline {
    private Logger logger = LoggerFactory.getLogger(this.getClass());


    private File file;

    public JsonAsFilePipeline(File file) {
        this.file = file;
    }

    public void process(ResultItems resultItems, Task task) {

        try {
            PrintWriter printWriter = new PrintWriter(new FileWriter(file));
            printWriter.write(JSON.toJSONString(resultItems.getAll()));
            printWriter.close();
        } catch (IOException var5) {
            this.logger.warn("write file error", var5);
        }

    }
}
